{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from nltk.corpus import stopwords\n",
    "STOPWORDS = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Put the hyparameters at the top like this to make it easier to change and edit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 5000\n",
    "embedding_dim = 64\n",
    "max_length = 200\n",
    "trunc_type = 'post'\n",
    "padding_type = 'post'\n",
    "oov_tok = '<OOV>'\n",
    "training_portion = .8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's define two lists that containing articles and labels. In the meantime, we remove stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2225\n",
      "2225\n"
     ]
    }
   ],
   "source": [
    "articles = []\n",
    "labels = []\n",
    "\n",
    "with open(\"bbc-text.csv\", 'r') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    next(reader)\n",
    "    for row in reader:\n",
    "        labels.append(row[0])\n",
    "        article = row[1]\n",
    "        for word in STOPWORDS:\n",
    "            token = ' ' + word + ' '\n",
    "            article = article.replace(token, ' ')\n",
    "            article = article.replace(' ', ' ')\n",
    "        articles.append(article)\n",
    "print(len(labels))\n",
    "print(len(articles))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are only 2,225 articles in the data. Then we split into training set and validation set, according to the parameter we set earlier, 80% for training, 20% for validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1780\n",
      "1780\n",
      "1780\n",
      "445\n",
      "445\n"
     ]
    }
   ],
   "source": [
    "train_size = int(len(articles) * training_portion)\n",
    "\n",
    "train_articles = articles[0: train_size]\n",
    "train_labels = labels[0: train_size]\n",
    "\n",
    "validation_articles = articles[train_size:]\n",
    "validation_labels = labels[train_size:]\n",
    "\n",
    "print(train_size)\n",
    "print(len(train_articles))\n",
    "print(len(train_labels))\n",
    "print(len(validation_articles))\n",
    "print(len(validation_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenizer does all the heavy lifting for us. In our articles that it was tokenizing, it will take 5,000 most common words. oov_token is to put a special value in when an unseen word is encountered. This means I want \"OOV\" in bracket to be used to for words that are not in the word index. \"fit_on_text\" will go through all the text and create dictionary like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n",
    "tokenizer.fit_on_texts(train_articles)\n",
    "word_index = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see that \"OOV\" in bracket is number 1, \"said\" is number 2, \"mr\" is number 3, and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<OOV>': 1,\n",
       " 'said': 2,\n",
       " 'mr': 3,\n",
       " 'would': 4,\n",
       " 'year': 5,\n",
       " 'also': 6,\n",
       " 'people': 7,\n",
       " 'new': 8,\n",
       " 'us': 9,\n",
       " 'one': 10}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(list(word_index.items())[0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This process cleans up our text, lowercase, and remove punctuations."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After tokenization, the next step is to turn thoes tokens into lists of sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sequences = tokenizer.texts_to_sequences(train_articles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the 11th article in the training data that has been turned into sequences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2432, 1, 225, 4995, 22, 642, 587, 225, 4995, 1, 1, 1662, 1, 1, 2432, 22, 565, 1, 1, 140, 278, 1, 140, 278, 796, 822, 662, 2308, 1, 1144, 1693, 1, 1720, 4996, 1, 1, 1, 1, 1, 4737, 1, 1, 122, 4513, 1, 2, 2875, 1506, 352, 4738, 1, 52, 341, 1, 352, 2173, 3961, 41, 22, 3794, 1, 1, 1, 1, 543, 1, 1, 1, 835, 631, 2367, 347, 4739, 1, 365, 22, 1, 787, 2368, 1, 4301, 138, 10, 1, 3665, 682, 3531, 1, 22, 1, 414, 822, 662, 1, 90, 13, 633, 1, 225, 4995, 1, 599, 1, 1693, 1021, 1, 4997, 807, 1863, 117, 1, 1, 1, 2975, 22, 1, 99, 278, 1, 1608, 4998, 543, 492, 1, 1446, 4740, 778, 1320, 1, 1860, 10, 33, 642, 319, 1, 62, 478, 565, 301, 1507, 22, 479, 1, 1, 1665, 1, 797, 1, 3067, 1, 1365, 6, 1, 2432, 565, 22, 2972, 4734, 1, 1, 1, 1, 1, 850, 39, 1824, 675, 297, 26, 979, 1, 882, 22, 361, 22, 13, 301, 1507, 1343, 374, 20, 63, 883, 1096, 4302, 247]\n"
     ]
    }
   ],
   "source": [
    "print(train_sequences[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we train neural networks for NLP, we need sequences to be in the same size, that's why we use padding. Our max_length is 200, so we use pad_sequences to make all of our articles the same length which is 200 in my example. That's why you see that the 1st article was 426 in length, becomes 200, the 2nd article was 192 in length, becomes 200, and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_padded = pad_sequences(train_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "425\n",
      "200\n",
      "192\n",
      "200\n",
      "186\n",
      "200\n"
     ]
    }
   ],
   "source": [
    "print(len(train_sequences[0]))\n",
    "print(len(train_padded[0]))\n",
    "\n",
    "print(len(train_sequences[1]))\n",
    "print(len(train_padded[1]))\n",
    "\n",
    "print(len(train_sequences[10]))\n",
    "print(len(train_padded[10]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addtion, there is padding type and truncating type, there are all \"post\". Means for example, for the 11th article, it was 186 in length, we padded to 200, and we padded at the end, add 14 zeros."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2432, 1, 225, 4995, 22, 642, 587, 225, 4995, 1, 1, 1662, 1, 1, 2432, 22, 565, 1, 1, 140, 278, 1, 140, 278, 796, 822, 662, 2308, 1, 1144, 1693, 1, 1720, 4996, 1, 1, 1, 1, 1, 4737, 1, 1, 122, 4513, 1, 2, 2875, 1506, 352, 4738, 1, 52, 341, 1, 352, 2173, 3961, 41, 22, 3794, 1, 1, 1, 1, 543, 1, 1, 1, 835, 631, 2367, 347, 4739, 1, 365, 22, 1, 787, 2368, 1, 4301, 138, 10, 1, 3665, 682, 3531, 1, 22, 1, 414, 822, 662, 1, 90, 13, 633, 1, 225, 4995, 1, 599, 1, 1693, 1021, 1, 4997, 807, 1863, 117, 1, 1, 1, 2975, 22, 1, 99, 278, 1, 1608, 4998, 543, 492, 1, 1446, 4740, 778, 1320, 1, 1860, 10, 33, 642, 319, 1, 62, 478, 565, 301, 1507, 22, 479, 1, 1, 1665, 1, 797, 1, 3067, 1, 1365, 6, 1, 2432, 565, 22, 2972, 4734, 1, 1, 1, 1, 1, 850, 39, 1824, 675, 297, 26, 979, 1, 882, 22, 361, 22, 13, 301, 1507, 1343, 374, 20, 63, 883, 1096, 4302, 247]\n"
     ]
    }
   ],
   "source": [
    "print(train_sequences[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2432    1  225 4995   22  642  587  225 4995    1    1 1662    1    1\n",
      " 2432   22  565    1    1  140  278    1  140  278  796  822  662 2308\n",
      "    1 1144 1693    1 1720 4996    1    1    1    1    1 4737    1    1\n",
      "  122 4513    1    2 2875 1506  352 4738    1   52  341    1  352 2173\n",
      " 3961   41   22 3794    1    1    1    1  543    1    1    1  835  631\n",
      " 2367  347 4739    1  365   22    1  787 2368    1 4301  138   10    1\n",
      " 3665  682 3531    1   22    1  414  822  662    1   90   13  633    1\n",
      "  225 4995    1  599    1 1693 1021    1 4997  807 1863  117    1    1\n",
      "    1 2975   22    1   99  278    1 1608 4998  543  492    1 1446 4740\n",
      "  778 1320    1 1860   10   33  642  319    1   62  478  565  301 1507\n",
      "   22  479    1    1 1665    1  797    1 3067    1 1365    6    1 2432\n",
      "  565   22 2972 4734    1    1    1    1    1  850   39 1824  675  297\n",
      "   26  979    1  882   22  361   22   13  301 1507 1343  374   20   63\n",
      "  883 1096 4302  247    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0]\n"
     ]
    }
   ],
   "source": [
    "print(train_padded[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And for the 1st article, it was 426 in length, we truncated to 200, and we truncated at the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[91, 160, 1141, 1106, 49, 979, 755, 1, 89, 1304, 4288, 129, 175, 3654, 1215, 1195, 1577, 42, 7, 893, 91, 1, 334, 85, 20, 14, 130, 3262, 1216, 2422, 570, 451, 1376, 58, 3378, 3521, 1660, 8, 921, 730, 10, 844, 1, 9, 598, 1578, 1107, 395, 1939, 1106, 731, 49, 538, 1398, 2011, 1623, 134, 249, 113, 2356, 795, 4980, 980, 584, 10, 3955, 3956, 921, 2563, 129, 344, 175, 3654, 1, 1, 39, 62, 2868, 28, 9, 4722, 18, 1305, 136, 416, 7, 143, 1423, 71, 4500, 436, 4981, 91, 1107, 77, 1, 82, 2012, 53, 1, 91, 6, 1008, 609, 89, 1304, 91, 1963, 131, 137, 420, 9, 2869, 38, 152, 1235, 89, 1304, 4723, 7, 436, 4981, 3154, 6, 2493, 1, 431, 1126, 1, 1424, 571, 1217, 1901, 1, 766, 9, 538, 1398, 2011, 134, 2068, 400, 845, 1964, 1601, 34, 1716, 2870, 1, 1, 2423, 244, 9, 2625, 82, 732, 6, 1173, 1196, 152, 720, 591, 1, 124, 28, 1305, 1689, 432, 83, 933, 115, 20, 14, 18, 3155, 1, 37, 1485, 1, 23, 37, 87, 335, 2357, 37, 467, 255, 1964, 1359, 328, 1, 299, 732, 1174, 18, 2871, 1716, 1, 294, 756, 1074, 395, 2013, 387, 431, 2013, 2, 1360, 1, 1716, 2167, 67, 1, 1, 1717, 249, 1661, 3060, 1175, 395, 41, 878, 246, 2794, 345, 53, 548, 400, 2, 1, 1, 655, 1361, 203, 91, 3957, 91, 90, 42, 7, 320, 395, 77, 893, 1, 91, 1106, 400, 538, 9, 845, 2423, 11, 38, 1, 995, 514, 483, 2069, 160, 572, 1, 128, 7, 320, 77, 893, 1218, 1126, 1465, 346, 54, 2215, 1219, 741, 92, 256, 274, 1019, 71, 623, 346, 2424, 756, 1216, 2358, 1718, 1, 3783, 3522, 1, 1126, 2013, 177, 371, 1399, 77, 53, 548, 105, 1141, 3, 1, 1047, 93, 2963, 1, 2626, 1, 102, 902, 440, 452, 2, 3, 1, 2872, 451, 1425, 43, 77, 429, 31, 8, 1019, 921, 1, 2563, 30, 1, 91, 1690, 879, 89, 1304, 91, 1963, 1, 30, 8, 1624, 1, 1, 4289, 1579, 4288, 656, 1, 3784, 1008, 572, 4290, 2868, 10, 880, 656, 58, 1, 1262, 1, 1, 91, 1554, 934, 4722, 1, 577, 4105, 10, 9, 235, 2011, 91, 134, 1, 95, 656, 3263, 1, 58, 520, 673, 2627, 3784, 4982, 3379, 483, 4724, 39, 4500, 1, 91, 1747, 673, 269, 116, 239, 2628, 354, 644, 58, 4106, 757, 3655, 4722, 146, 1, 400, 7, 71, 1748, 1107, 767, 910, 118, 584, 3380, 1316, 1578, 1, 1602, 7, 893, 77, 77]\n"
     ]
    }
   ],
   "source": [
    "print(train_sequences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  91  160 1141 1106   49  979  755    1   89 1304 4288  129  175 3654\n",
      " 1215 1195 1577   42    7  893   91    1  334   85   20   14  130 3262\n",
      " 1216 2422  570  451 1376   58 3378 3521 1660    8  921  730   10  844\n",
      "    1    9  598 1578 1107  395 1939 1106  731   49  538 1398 2011 1623\n",
      "  134  249  113 2356  795 4980  980  584   10 3955 3956  921 2563  129\n",
      "  344  175 3654    1    1   39   62 2868   28    9 4722   18 1305  136\n",
      "  416    7  143 1423   71 4500  436 4981   91 1107   77    1   82 2012\n",
      "   53    1   91    6 1008  609   89 1304   91 1963  131  137  420    9\n",
      " 2869   38  152 1235   89 1304 4723    7  436 4981 3154    6 2493    1\n",
      "  431 1126    1 1424  571 1217 1901    1  766    9  538 1398 2011  134\n",
      " 2068  400  845 1964 1601   34 1716 2870    1    1 2423  244    9 2625\n",
      "   82  732    6 1173 1196  152  720  591    1  124   28 1305 1689  432\n",
      "   83  933  115   20   14   18 3155    1   37 1485    1   23   37   87\n",
      "  335 2357   37  467  255 1964 1359  328    1  299  732 1174   18 2871\n",
      " 1716    1  294  756]\n"
     ]
    }
   ],
   "source": [
    "print(train_padded[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we do the same for the validation sequences. Note that we should expect more out of vocabulary words from validation articles because word index were derived from the training articles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "445\n",
      "(445, 200)\n"
     ]
    }
   ],
   "source": [
    "validation_sequences = tokenizer.texts_to_sequences(validation_articles)\n",
    "validation_padded = pad_sequences(validation_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n",
    "\n",
    "print(len(validation_sequences))\n",
    "print(validation_padded.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are going to look at the labels. because our labels are text, so we will tokenize them, when training, labels are expected to be numpy arrays. So we will turn list of labels into numpy arrays like so:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'tech', 'politics', 'sport', 'business', 'entertainment'}\n"
     ]
    }
   ],
   "source": [
    "print(set(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_tokenizer = Tokenizer()\n",
    "label_tokenizer.fit_on_texts(labels)\n",
    "\n",
    "training_label_seq = np.array(label_tokenizer.texts_to_sequences(train_labels))\n",
    "validation_label_seq = np.array(label_tokenizer.texts_to_sequences(validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4]\n",
      "[2]\n",
      "[1]\n",
      "(1780, 1)\n",
      "[5]\n",
      "[4]\n",
      "[3]\n",
      "(445, 1)\n"
     ]
    }
   ],
   "source": [
    "print(training_label_seq[0])\n",
    "print(training_label_seq[1])\n",
    "print(training_label_seq[2])\n",
    "print(training_label_seq.shape)\n",
    "\n",
    "print(validation_label_seq[0])\n",
    "print(validation_label_seq[1])\n",
    "print(validation_label_seq[2])\n",
    "print(validation_label_seq.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before training deep neural network, we want to explore what our original article and article after padding look like. Running the following code, we explore the 11th article, we can see that some words become \"OOV\", because they did not make to the top 5,000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "berlin <OOV> anti nazi film german movie anti nazi <OOV> <OOV> drawn <OOV> <OOV> berlin film festival <OOV> <OOV> final days <OOV> final days member white rose movement <OOV> 21 arrested <OOV> brother hans <OOV> <OOV> <OOV> <OOV> <OOV> tyranny <OOV> <OOV> director marc <OOV> said feeling responsibility keep legacy <OOV> going must <OOV> keep ideas alive added film drew <OOV> <OOV> <OOV> <OOV> trial <OOV> <OOV> <OOV> east germany secret police discovery <OOV> behind film <OOV> worked closely <OOV> relatives including one <OOV> sisters ensure historical <OOV> film <OOV> members white rose <OOV> group first started <OOV> anti nazi <OOV> summer <OOV> arrested dropped <OOV> munich university calling day <OOV> <OOV> <OOV> regime film <OOV> six days <OOV> arrest intense trial saw <OOV> initially deny charges ended <OOV> appearance one three german films <OOV> top prize festival south african film version <OOV> <OOV> opera <OOV> shot <OOV> town <OOV> language also <OOV> berlin festival film entitled u <OOV> <OOV> <OOV> <OOV> <OOV> story set performed 40 strong music theatre <OOV> debut film performance film first south african feature 25 years second nominated golden bear award ? ? ? ? ? ? ? ? ? ? ? ? ? ?\n",
      "---\n",
      "berlin cheers anti-nazi film german movie anti-nazi resistance heroine drawn loud applause berlin film festival.  sophie scholl - final days portrays final days member white rose movement. scholl  21  arrested beheaded brother  hans  1943 distributing leaflets condemning  abhorrent tyranny  adolf hitler. director marc rothemund said:  feeling responsibility keep legacy scholls going.   must somehow keep ideas alive   added.  film drew transcripts gestapo interrogations scholl trial preserved archive communist east germany secret police. discovery inspiration behind film rothemund  worked closely surviving relatives  including one scholl sisters  ensure historical accuracy film. scholl members white rose resistance group first started distributing anti-nazi leaflets summer 1942. arrested dropped leaflets munich university calling  day reckoning  adolf hitler regime. film focuses six days scholl arrest intense trial saw scholl initially deny charges ended defiant appearance. one three german films vying top prize festival.  south african film version bizet tragic opera carmen shot cape town xhosa language also premiered berlin festival. film entitled u-carmen ekhayelitsha carmen khayelitsha township story set. performed 40-strong music theatre troupe debut film performance. film first south african feature 25 years second nominated golden bear award.\n"
     ]
    }
   ],
   "source": [
    "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
    "\n",
    "def decode_article(text):\n",
    "    return ' '.join([reverse_word_index.get(i, '?') for i in text])\n",
    "print(decode_article(train_padded[10]))\n",
    "print('---')\n",
    "print(train_articles[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can implement LSTM. Here is my code that I build a tf.keras.Sequential model and start with an embedding layer. An embedding layer stores one vector per word. When called, it converts the sequences of word indices into sequences of vectors. After training, words with similar meanings often have the similar vectors.\n",
    "\n",
    "Next is how to implement LSTM in code. The Bidirectional wrapper is used with a LSTM layer, this propagates the input forwards and backwards through the LSTM layer and then concatenates the outputs. This helps LSTM to learn long term dependencies. We then fit it to a dense neural network to do classification.\n",
    "\n",
    "This index-lookup is much more efficient than the equivalent operation of passing a one-hot encoded vector through a tf.keras.layers.Dense layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, None, 64)          320000    \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 128)               66048     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 6)                 390       \n",
      "=================================================================\n",
      "Total params: 394,694\n",
      "Trainable params: 394,694\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    # Add an Embedding layer expecting input vocab of size 5000, and output embedding dimension of size 64 we set at the top\n",
    "    tf.keras.layers.Embedding(vocab_size, embedding_dim),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim)),\n",
    "#    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n",
    "    # use ReLU in place of tanh function since they are very good alternatives of each other.\n",
    "    tf.keras.layers.Dense(embedding_dim, activation='relu'),\n",
    "    # Add a Dense layer with 6 units and softmax activation.\n",
    "    # When we have multiple outputs, softmax convert outputs layers into a probability distribution.\n",
    "    tf.keras.layers.Dense(6, activation='softmax')\n",
    "])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our model summay, we have our embeddings, our Bidirectional contains LSTM, followed by two dense layers. The output from Bidirectional is 128, because it doubled what we put in LSTM. We can also stack LSTM layer but I found the results worse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1780 samples, validate on 445 samples\n",
      "Epoch 1/10\n",
      "1780/1780 - 10s - loss: 1.6322 - accuracy: 0.2635 - val_loss: 1.4729 - val_accuracy: 0.2674\n",
      "Epoch 2/10\n",
      "1780/1780 - 5s - loss: 1.0612 - accuracy: 0.5809 - val_loss: 0.7554 - val_accuracy: 0.7393\n",
      "Epoch 3/10\n",
      "1780/1780 - 5s - loss: 0.3791 - accuracy: 0.8685 - val_loss: 0.3497 - val_accuracy: 0.8809\n",
      "Epoch 4/10\n",
      "1780/1780 - 5s - loss: 0.1476 - accuracy: 0.9556 - val_loss: 0.2603 - val_accuracy: 0.9146\n",
      "Epoch 5/10\n",
      "1780/1780 - 5s - loss: 0.0444 - accuracy: 0.9910 - val_loss: 0.3338 - val_accuracy: 0.9101\n",
      "Epoch 6/10\n",
      "1780/1780 - 5s - loss: 0.0185 - accuracy: 0.9961 - val_loss: 0.2438 - val_accuracy: 0.9326\n",
      "Epoch 7/10\n",
      "1780/1780 - 5s - loss: 0.0042 - accuracy: 1.0000 - val_loss: 0.2118 - val_accuracy: 0.9438\n",
      "Epoch 8/10\n",
      "1780/1780 - 5s - loss: 0.0016 - accuracy: 1.0000 - val_loss: 0.2476 - val_accuracy: 0.9371\n",
      "Epoch 9/10\n",
      "1780/1780 - 5s - loss: 0.0011 - accuracy: 1.0000 - val_loss: 0.2752 - val_accuracy: 0.9371\n",
      "Epoch 10/10\n",
      "1780/1780 - 5s - loss: 7.9578e-04 - accuracy: 1.0000 - val_loss: 0.2882 - val_accuracy: 0.9348\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "history = model.fit(train_padded, training_label_seq, epochs=num_epochs, validation_data=(validation_padded, validation_label_seq), verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXyU1b348c83C1lYAwlrgARFdiIQNlFB0Rar4oJsLq24/bpgXXpbl7Zqrbe3t7e2V++1vcUWlaqgggtaqxVJUBaBsAgCypKELEASAgQC2Sbz/f3xDCGBLJOQyWQy3/frNa+Z55nzPPPNKOc7zznPOUdUFWOMMcErxN8BGGOM8S9LBMYYE+QsERhjTJCzRGCMMUHOEoExxgS5MH8H0FixsbGakJDg7zCMMSagbNq06bCqxtX2XsAlgoSEBNLS0vwdhjHGBBQR2V/Xe9Y0ZIwxQc4SgTHGBDlLBMYYE+QsERhjTJCzRGCMMUHOZ4lARBaKSL6IfFXH+yIiz4vIXhHZJiKjfRWLMcaYuvnyiuBlYFo9718DDPQ87gP+7MNYjDHG1MFn4whU9TMRSainyA3AInXmwf5CRLqISC9VPeirmIwJBqqKy62UudyUVVQ6zy43Za5KyirqeH1WWVel299/hqnF1CE9SOrbpdnP688BZX2A7GrbOZ595yQCEbkP56qBfv36tUhwxjQXV6WbUxWVlJZXUlLhPE6V17JdUUlJeWU9FXW11w2UcTfDMiMi538O07y6d4psc4nAa6q6AFgAkJycbCvpmGZ37FQ5RSUVTsV8uoKu9lzqqaxPV9yl5We2S6tV5qfLVq/cKyob/79sWIgQERZCRHio8xwWQkRYKBHhzuuo8FC6RIV7tquVqaN8VZka5Wt/v11oCGGhdh9JMPFnIsgF+lbbjvfsM8bnylyVbMw4yqrd+azaXcDuvGKvj40ICyGqXShR4aFnnsND6RARRlyHiFrfi2pXy3Ytz5GeitwqYtOS/JkIlgPzRWQJMB4osv4B40uZh0+yancBq3YXsG5fISUVlbQLDWFsYgw3jupDj46RdVbQ1Svq0BBrMzFti88SgYgsBqYAsSKSAzwJhAOo6v8BHwLfAfYCp4B5vorFBKeTZS6+SC+sqvz3F54CIKFbNLOS45k8KI4JA7oR3S4gWkiN8Rlf3jU0t4H3FfiRrz7fBB9V5Zu8E6z6xqn40zKPUl7pJio8lEsu6MbdlyZy+cA4EmLb+ztUY1oV+ylkAlrRqQpW7z1c1dafd7wMgEE9OnLnpAQmXxRHckIMEWGhfo7UmNbLEoEJKG63sj23qKq5Z0vWUdwKnSLDuGxgHJMviuOyi2Lp1TnK36EaEzAsEZhWL/9EKZ/vPsyq3QWs3nuYIyfLEYGRfToz/4oLmTwojqT4LnanjTFNZInAtDoVlW427z9a9at/x4HjAMR2aMeUi+KYPCiOSy+MpVuHCD9HakzbYInAtAo5R0/x2W6nrX/N3kKKy1yEhghj+sfw028PYvJFcQzt1YkQu3XTmGZnicD4zalyF39K2cdHOw6xN98Z0NWnSxTXJ/Vm8kVxXHJhNzpFhvs5SmPaPksExi827T/KT97cyv4jp7j0wljmjO3LlEFxXBDXAbFJboxpUZYITIsqd7l5/tM9/Cl1L706R7H43glMGNDN32EZE9QsEZgWsyfvBA++sZUdB44zc0w8T1w/lI7W9ON/lRVwsgCK853nkwVQfhIiOkFEB4jo6DzadTzzOjzKpidtQywRGJ9zu5WFazL43cff0DEijL/cMYZvD+vp77DatoqSmhV7cT6czIfigmrPntclRxt/fgn1JIlOniRRLWFU31/jvToSS1i75v/7TaNYIjA+lXP0FP/21pd8kX6Eq4b04LczRhBrt302niqUnainYs+Hk4fP7Cs/Uft5IjpB+zjo0B3iBkHiZc726X3tu0OHOAhvD+XFzmeWnaj2+jiUVdtfdsL5rLITUHoMinJq7vdGaETNBBHR6UzyCIsAWsGVhwiEhkNIGISEQ0hoze3QsLrfq3Pbs6/GsXVtn34d7pMrMUsExidUlWWbc/nV8h24VfndjJHMTI63juC6FOdD3ldweA8U53kq+4Kalb2rtPZjo7p6KvE46D3Kqczbx9as2Nt73g+PbERQPc7vb3K7oeJktaRR7EkkZyeWEzWTS3kxFB+Cwj3gKj+/GJqLVoLb5TSjuSvBXeG81sqWjePaZ2HsPc1+WksEptkVFpfx+Dvb+XhHHuMSuvLsrCT6do32d1itQ0UJFHwNeTsgb6dT+eftgFOHz5SRUKciP12JdxtYe8XeoTtEd3N+JbZGISFnfuW3VapOgqhKEi7vtr0q60k6Ve9VQJ8xPvkzLBGYZvXprjweWbad4yUVPHbNYO65bEBwzt/vdkNR1rkV/pF9oJ71gMOioPsQGDQNegyH7kMhbrDzyz3EpssICKebjELDnQ70AGWJwDSL4jIXz3ywkyUbsxncsyN/v3scQ3p18ndYLaPkGOTv9FT6nkf+rppt5DGJ0GMYDL/Zee4+DLomOu3FxviZJQJz3jZmHuHhN7eSc7SE70++gIeuHtg2p32urIDCvWdV+DuhKPtMmcguTkV/8dwzFX73IU5nqDGtlCUC02Rlrkr++Mke/vLZPuJjonjz/01kbEJX7w4uPwn710LmanCVOZ2YYVG1PNe2r/p7URDarnnvpFB1OmzzvvI06+yA/B1Q8A1UejovQ8IgdhD0mwA97j7TtNOpt91fbwKOJQLTJF8fOs6DS7by9aETzB3Xl59fO5QOEfX87+SuhANbID0F9qVC9nqn8yu0nVOpu0rOVLKNJk5CCIus+VzbvhrP0WeSSkgIHN7rVP75O+FU4ZnTd+wNPYbCBVeeqfBjL7L7302b4dNEICLTgOeAUOCvqvrbs97vDywE4oAjwO2qmuPLmMz5qXQrf/08nWf/tZtOUeH87XvJTB1Sy22GqnAk3VPxp0Dm51Ba5LzXcyRM/CEMmAL9Jp7pZHNXOrdIVpQ4j9OvazyfgopSJ3Gc81zbvlLnVswKz7Gu0jPvnZ14wqOdSn7wtWcq/B7DINrLqxxjApQvF68PBV4ArgZygI0islxVd1Yr9ntgkaq+IiJXAv8B3OGrmMz5yT5yip+8+SUbMo8wbVhP/v2m4TXXBDh5GDJWORV/+irnrhmAzv1g6A1OxZ842bkVsjYhodCuvfNoCVWJp9RJCh162N06Jij58opgHLBXVdMBRGQJcANQPREMBR72vE4B3vVhPKaJVJW30nL41fs7CBHh2ZlJ3Dy6D+IqhX0rPRV/Khza5hwQ2RkSL4dLH4ABV0DXAa2z3bylE48xrZQvE0EfoNrtFOQA488q8yVwM07z0U1ARxHppqqF1QuJyH3AfQD9+vXzWcDmXAUnynjs7e2s2JXHJYld+ONkoUfBm7AoBbLWQ2WZM/y93wS48hcw4ErofbHdFmlMAPF3Z/G/Af8rIncCnwG5wDljtlV1AbAAIDk5WVsywGD28Y5DvLBsBUkVW1jZL5PEY5uQJZ4JynoMh3H3Or/4+0+0X9XGBDBfJoJcoG+17XjPviqqegDnigAR6QDMUNVjPozJNOTUEUp2p7B11TsMLlzP8pB8p6u/pDcM+o5T8Q+Y7ExvYIxpE3yZCDYCA0UkEScBzAFurV5ARGKBI6rqBh7DuYPItKSKUudWznSnnV8PbCUKZbhGkddtLK5xPyHswishdmDrbOc3xpw3nyUCVXWJyHzgY5zflAtVdYeIPA2kqepyYArwHyKiOE1DP/JVPOYsO96BzYtg/zpwlaAhYWRHD2Opawb7OiRz1+xbGJMY5+8ojTEtQFQDq8k9OTlZ09LS/B1GYNu+FJbdDV0vgIHfYn/nsfz4i2i+zHdz+4R+PP6dIUS383f3kTGmOYnIJlVNru09+9cebDI+g3d/AP0nUXnbMv5vTS7//Y/dxESH8/K8kUwZZG3/xgQbSwTBJG8HLLkNul5A9rde5MG/bWHT/qNcO6IXz9w4nJj2NmWCMcHIEkGwOJYNr86Adh2ovPUtbv/rbo6eLOe5ORczPam3rRxmTBCzRBAMSo7Ca7c4M37e9RGf5Iazv/AUf75tNNeM6OXv6IwxfmYTq7R1FaWw+FZnArg5r0GPYSxcnUF8TBTfGtbT39EZY1oBSwRtmdsN79wHWWvhpv+DxMvZnlPEhswj3HlJQnAuIWmMOYclgrZKFT5+DHa+B9/+DQyfAcBLazJo3y6UWWP7NnACY0ywsETQVq39H1j/fzBxPkx0xunlHy/l/W0HmJncl06R4X4O0BjTWlgiaIu2vQWf/BKG3QxX/7pq96tf7MflVu68JMF/sRljWh1LBG1NeqozYCzhMqdfwLPQSmlFJa+uz2Lq4B4kxNpMocaYMywRtCWHtsOS250J4ma/CmFnVg9bvvUAR06Wc9elCf6LzxjTKlkiaCuOZcGrt0BkJ7htKUR1qXpLVVm4JoPBPTsycUA3PwZpjGmNLBG0BaeOOKOGXSVw+zLo3KfG22v3FfL1oRPcdWmijSA2xpzDRhYHuooSWDwXjmbCHe9C9yHnFFm4OoNu7dsxPal3y8dnjGn17IogkLkr4e17nYVlbl4ACZPOKZJx+CSffp3PbRP6Exlu6wgbY85lVwSBShX++Qjseh+m/ScMu6nWYi+vyaBdaAi3T+jXwgEaYwKFXREEqjX/DRtfhEvuhwnfr7VIUUkFb23K4fqk3nTvGNnCARpjAoUlgkD05RJY8RQMvwWuerrOYm9uzOZUeSXzJiW0WGjGmMDj00QgItNE5BsR2Ssij9byfj8RSRGRLSKyTUS+48t42oR9K+G9H0Hi5XDjn6oGjJ3NVenm5bWZjE/syvA+nVs4SGNMIPFZIhCRUOAF4BpgKDBXRIaeVewXwJuqOgqYA/zJV/G0CQe/hDfugLjB5wwYO9u/duaRe6yEuy5NbMEAjTGByJdXBOOAvaqarqrlwBLghrPKKNDJ87ozcMCH8QS2o/vhtZkQFeMMGIus/1f+wtUZ9O0axVVDerRQgMaYQOXLRNAHyK62nePZV91TwO0ikgN8CNxf24lE5D4RSRORtIKCAl/E2rpVDRgrcwaMdap/VbEvs4+Rtv8od16SaGsOGGMa5O/O4rnAy6oaD3wH+LuInBOTqi5Q1WRVTY6Li2vxIP2qogRen+1MITF3CcQNavCQl9Zk0CEijFnJ8S0QoDEm0PkyEeQC1Vc/iffsq+5u4E0AVV0HRAKxPowpsLgrYendkLMRZvwV+k9s8JC846V8sO0gs5L70tHWHDDGeMGXiWAjMFBEEkWkHU5n8PKzymQBUwFEZAhOIgjCtp9aqMKHP4Vv/gHX/A6GTvfqsL+v20+l2poDxhjv+SwRqKoLmA98DOzCuTtoh4g8LSKna7WfAPeKyJfAYuBOVVVfxRRQPn8W0v4Gkx6E8fd5dUhpRSWvrd/P1UN60K9btI8DNMa0FT6dYkJVP8TpBK6+74lqr3cC506QE+y2vAYrfw0jZ8PUJ70+7J0tuRw9VWG3jBpjGsXfncXmbHtXwPs/hgFTYPr/1jlg7GyqysLVGQzt1YnxiV19GqIxpm2xRNCaHNgCb3zXmUp61t8hrJ3Xh67ee5g9+cW25oAxptEsEbQWRzKcAWPR3TwDxjo1fEw1C1dnENshguuT6h9jYIwxZ7NE0BqcLHQGjLldzoCxjj0bdfi+gmJSvingjgn9iQizNQeMMY1j6xH4W/kpeH0WHM+F774HcRc1+hQvedYcuM3WHDDGNIElAn+qdMHSu+DAZqdPoN+ERp/i2Klylm3K5YaLexPboe5J6Iwxpi6WCPxFFf7xMOz+J1z7LAy5rkmnWbIxm5KKSuZNsltGjTFNY30E/vLZf8HmV+Cyn8DYe5p0iopKN6+szWTigG4M7d24zmVjjDnNEoE/bP47pPw7JM2FK3/Z5NN8vOMQB4tKbQCZMea8WCJoaftWwvsPwAVTYfr/wHnc879wdQb9u0UzdXD3ZgzQGBNsLBG0tM//AF36waxXILTps4NuyTrK5qxjzLskgRBbc8AYcx4sEbSk8pOQvR6GXA8RHc/rVAvXZNIxIoxbkvs2XNgYY+phiaAl7V8HleXOPELn4WBRCR9uP8jssX3pEGE3fhljzo8lgpaUngKhEdD/kvM6zaJ1+1FVvmdrDhhjmoElgpaUvgr6jYfwqCafoqS8ktfXZ/GtoT3p29XWHDDGnD9LBC2lOB/ytp93s9DbW3IoKrE1B4wxzccSQUvJ+Mx5HnBFk0/hdjtrDozo05mxCTHNFJgxJthZImgp+1Igsgv0SmryKT7fe5h9BSe569IEW3PAGNNsfJoIRGSaiHwjIntF5NFa3v+jiGz1PHaLyDFfxuM3qpCeComXQ0jTp4n+2+oM4jpGcO2I3s0XmzEm6HmVCETkbRG5VkS8ThwiEgq8AFwDDAXmisjQ6mVU9SFVvVhVLwb+B3jb+9ADSOE+OJ4DFzS9WWhP3gk+213Adyf0p12YXcgZY5qPtzXKn4BbgT0i8lsRGeTFMeOAvaqarqrlwBLghnrKzwUWexlPYElPcZ4HTGnyKV5am0m7sBBuHW9rDhhjmpdXiUBVV6jqbcBoIBNYISJrRWSeiNQ1T0IfILvado5n3zlEpD+QCKys4/37RCRNRNIKCgq8Cbl1SU91ppWIadqdPkdPlvP25hxuurgP3WzNAWNMM2tMU0834E7gHmAL8BxOYvikGeKYAyxV1cra3lTVBaqarKrJcXFxzfBxLajSBRmfO3cLNbGDd/HGLEor3My7NKF5YzPGGLxcmEZE3gEGAX8HrlfVg5633hCRtDoOywWqT4QT79lXmznAj7yJJeAc2AJlRU1uFqqodLNo7X4uvTCWwT1tzQFjTPPzdqKa51U1pbY3VDW5jmM2AgNFJBEnAczB6WeoQUQGAzHAOi9jCSzpqYBA4uQmHf7Prw5x6Hgpv7l5eLOGZYwxp3nbNDRURLqc3hCRGBH5YX0HqKoLmA98DOwC3lTVHSLytIhMr1Z0DrBEVbWRsQeG9FToNRLad2v0oarK31ZnkBjbnikX2ZoDxhjf8PaK4F5VfeH0hqoeFZF7ce4mqpOqfgh8eNa+J87afsrLGAJPWbEz7fTEenNmnTZnHePL7GM8fcMwW3PAGOMz3l4RhEq1oayeMQLtfBNSG5K1DtwVTZ5WYuGaDDpGhjFjdHwzB2aMMWd4e0XwEU7H8F882//Ps8/UJz3VmXa634RGH5p7rISPvjrE3Zcm0t7WHDDG+JC3NcwjOJX/DzzbnwB/9UlEbcm+FCcJNGHa6UXrMlFVvjuxf/PHZYwx1XiVCFTVDfzZ8zDeOJEH+TvgqqcafeipcheL12cxbXhP4mNszQFjjG95O45gIPAfOHMGRZ7er6oDfBRX4MtY5TwPmNLoQ5dtzuV4qYu7bc0BY0wL8Laz+CWcqwEXcAWwCHjVV0G1CempEBUDPUc26jC3W3lpdQZJ8Z0Z3c/WHDDG+J63iSBKVT8FRFX3e275vNZ3YQW4qmmnJzd62ulVuwtIP3ySuy5NtDUHjDEtwtvO4jLPFNR7RGQ+zkjhDr4LK8Ad3gPHc2HATxt96MI1GfToFME1w3v5IDBjjDmXt1cEDwDRwI+BMcDtwPd8FVTAS091ngdMadRhu/NO8Pmew3x3YoKtOWCMaTENXhF4Bo/NVtV/A4qBeT6PKtClp0JMAnRtXGfvS2syiAgLYe44W3PAGNNyGvzZ6Zka+tIWiKVtqHRB5ueNvho4crKctzfncvPoPnRtb4O2jTEtx9s+gi0ishx4Czh5eqeqts2lJc/Hgc1QdrzRieD19fspc7m5a5LdMmqMaVneJoJIoBC4sto+pa2uMXw+mjDtdLnLzaJ1+7lsYCwDe3T0WWjGGFMbb0cWW7+At/alQK8kiO7q9SEfbj9I/oky/vOWxo05MMaY5uDtyOKXcK4AalDVu5o9okBWVgw5G2DifK8PUVUWrslgQFx7Jg8MsGU4jTFtgrdNQx9Uex0J3AQcaP5wAtz+teB2wQXeTzu9af9RtuUU8esbh9uaA8YYv/C2aWhZ9W0RWQys9klEgSw9BcIioa/3004vXJNBp8gwZozu48PAjDGmbk0dtTQQaHDtRBGZJiLfiMheEXm0jjKzRGSniOwQkdebGE/rkJ4K/SZCeGSDRQFyjp7io68OMXd8P6Lb2ZoDxhj/8LaP4AQ1+wgO4axRUN8xocALwNVADrBRRJar6s5qZQYCjwGTPMtfBu7CvCfyIH8njJzt9SGvrM1ERPjexATfxWWMMQ3wtmmoKfc0jgP2qmo6gIgsAW4AdlYrcy/wgqoe9XxOfhM+p3Vo5LQSxWUulmzM5prhPendpfEL1xhjTHPxqmlIRG4Skc7VtruIyI0NHNYHyK62nePZV91FwEUiskZEvhCRad7E0yqlp0JUV6+nnV62KYcTpS7usjUHjDF+5m0fwZOqWnR6Q1WPAU82w+eH4fQ3TAHmAi+KSJezC4nIfSKSJiJpBQUFzfCxzUzV6SgeMBlCGv5K3W7lpTUZXNy3i605YIzxO28TQW3lGmpWygX6VtuO9+yrLgdYrqoVqpoB7MZJDDWo6gJVTVbV5Li4Vniv/eHdcOKg181CKd/kk1l4yq4GjDGtgreJIE1E/iAiF3gefwA2NXDMRmCgiCSKSDtgDrD8rDLv4lwNICKxOE1F6V5H31pU9Q94N35g4ZoMenaK5JrhPX0XkzHGeMnbRHA/UA68ASwBSoEf1XeAqrqA+cDHwC7gTVXdISJPi8h0T7GPgUIR2QmkAD9V1cLG/xl+ti8FYhIhpn+DRb8+dJw1ewv57iX9CQ+1NQeMMf7n7V1DJ4FaxwE0cNyHwIdn7Xui2msFHvY8AlNlBWSuhhG3eFX85TWZRIaHcKutOWCMaSW8vWvok+qduCISIyIf+y6sAJK7GcpPeDWtxInSCt7beoDpSb3pEm1rDhhjWgdv2yZiPXcKAeC57z9wB381p/QUQCDhsgaLLv/yACUVlbYCmTGmVfE2EbhFpKr2EpEEapmNNCilp0Lvi72adnrxhiwG9+zIxX3PuUPWGGP8xtsJbn4OrBaRVYAAlwH3+SyqQFF2AnI2wiU/brDo9pwivso9zq+mD0PEZhk1xrQe3nYWfyQiyTiV/xac2z5LfBlYQMhc40w7PWBKg0UXb8wiIiyEG0fZLKPGmNbF20nn7gEewBkUthWYAKyj5tKVwSc91TPt9Ph6i50sc/HellyuG9mbzlHhLRObMcZ4yds+ggeAscB+Vb0CGAUcq/+QIJCeCv0vaXDa6Q+2HeBkeSVzx/Wtt5wxxviDt4mgVFVLAUQkQlW/Bgb5LqwAcPwgFOzyqlno9Q3ZDOzegTH9bV4hY0zr420iyPGMI3gX+ERE3gP2+y6sAJCxynluYFqJnQeO82X2MeaO62edxMaYVsnbzuKbPC+fEpEUoDPwkc+iCgT7UiC6G/QYXm+xJRuzaBcWws22FKUxppVq9PqIqrrKF4EEFFWnfyCx/mmnS8oreWdLLt8Z3tNGEhtjWi2b9awpCr6B4kMNTivxj+0HOVHqYo6NJDbGtGKWCJoiPcV5HjCl3mKLN2QxILY94xMbHnVsjDH+YomgKdJToesA6FL3L/3deSfYtP+odRIbY1o9SwSNdXra6QbuFlq8IYvwULFOYmNMq2eJoLFy0qC8uN5modKKSt7enMu3h/WkW4eIFgvNGGOawhJBY6WngoRAYt3TTv/zq4MUlVTY4jPGmIBgiaCx0lOh9yiIqnuU8OIN2fTvFs2EAd1aLi5jjGkinyYCEZkmIt+IyF4ROWepSxG5U0QKRGSr53GPL+M5b6XHnWmnB0yps8je/GI2ZBxhzth+hIRYJ7ExpvVr9IAyb4lIKPACcDWQA2wUkeWquvOsom+o6nxfxdGs9q8Braw3EbyxMYuwEOGWMfEtFpYxxpwPX14RjAP2qmq6qpYDS4AbfPh5vpeeCmFRdU47XeaqZOmmHK4e2oO4jtZJbIwJDL5MBH2A7GrbOZ59Z5shIttEZKmI1DpPs4jcJyJpIpJWUFDgi1i9sy/FmXY6rPZK/uMdeRw9VWFrEhtjAoq/O4vfBxJUdSTwCfBKbYVUdYGqJqtqclxcXIsGWOX4ATj8Tb3NQks2ZBEfE8WlF8a2WFjGGHO+fJkIcoHqv/DjPfuqqGqhqpZ5Nv8KjPFhPOcn3TPXXh3zC2UcPsnafYXMGdvXOomNMQHFl4lgIzBQRBJFpB0wB1hevYCI9Kq2OR3Y5cN4zk96CkTHQvdhtb69ZGMWoSHCzGRbhcwYE1h8dteQqrpEZD7wMRAKLFTVHSLyNJCmqsuBH4vIdMAFHAHu9FU85+X0tNMDptQ67XS5y82yTTlcObg7PTrVv2ylMca0Nj5LBACq+iHw4Vn7nqj2+jHgMV/G0Czyd0FxXp39Ayt25XG4uNxGEhtjApK/O4sDQ3qq8zxgSq1vL96QRe/OkVx+kZ86so0x5jxYIvBGeip0uxC6nNv+n33kFJ/vOcyssX0JtU5iY0wAskTQEFe5Z9rpKbW+vWRjFiECs6yT2BgToCwRNCQ3DSpO1poIKirdvJWWw5RB3endJarFQzPGmOZgiaAhp6edTjh32umVX+eTf6LMRhIbYwKaJYKG7EuB3qMhqss5by3ekEWPThFcMcg6iY0xgcsSQX1KiyB3U63NQrnHSli1u4BZyX0JC7Wv0RgTuKwGq0+mZ9rpWqaVeGOjM5+edRIbYwKdJYL6pKdAeDTEj62x21Xp5q20bC4bGEffrtF+Cs4YY5qHJYL6pKfWOu30qt0FHCwq5dZxdjVgjAl8lgjqUpQLh3fDgHObhRZvyCK2QwRTh/TwQ2DGGNO8LBHUpY5pJQ4VlbLy63xmJscTbp3Expg2wGqyuqSnQvs46D60xu4307JxK8wZa81Cxpi2wRJBbeqYdrrSrbyxMZtLL4ylf7f2/orOGGOalSWC2uTvhJP55zQLfb6ngNxjJcyxTmJjTBtiiaA2dfQPLN6QRbf27fjW0J4tHZExxviMJYLa7EuBbgOhc3zVrvzjpXy6K58ZY+JpF2ZfmzGm7aIO70cAABIHSURBVLAa7Wyucti/5pyrgbc25eByq3USG2PaHJ8mAhGZJiLfiMheEXm0nnIzRERFJNmX8XglZyNUnKoxrYTbrSzZmMWEAV0ZENfBj8EZY0zz81kiEJFQ4AXgGmAoMFdEhtZSriPwALDeV7E0SnqKZ9rpS6t2rd1XSPaREptu2hjTJvnyimAcsFdV01W1HFgC3FBLuV8D/wmU+jAW76WnQp8xENm5atfiDVl0iQ7n28Osk9gY0/b4MhH0AbKrbed49lURkdFAX1X9R30nEpH7RCRNRNIKCgqaP9LTqqadPtMsdLi4jH/tPMSM0fFEhof67rONMcZP/NZZLCIhwB+AnzRUVlUXqGqyqibHxflwEZiMz0HdNTqKl23KoaJSmWtjB4wxbZQvE0EuUL32jPfsO60jMBxIFZFMYAKw3K8dxumpEN6+atppVWXJxmzGJsRwYfeOfgvLGGN8yZeJYCMwUEQSRaQdMAdYfvpNVS1S1VhVTVDVBOALYLqqpvkwpvqlp0LCJAhrB8AX6UfIOHySOWOtk9gY03b5LBGoqguYD3wM7ALeVNUdIvK0iEz31ec2WVEOFO6p0Sy0eEMWnSLDuHZkL7+FZYwxvhbmy5Or6ofAh2fte6KOslN8GUuDzppW4sjJcj766hC3ju9nncTGmDbNRhafti8F2nevmnb67c05lFe6bYI5Y0ybZ4kAwO0+M+20CKrK4g1ZjOrXhcE9O/k5OGOM8S2fNg0FjPydcOpwVbNQ2v6j7Cs4ye9uGenXsIwJBBUVFeTk5FBa2jrGhAa7yMhI4uPjCQ8P9/oYSwTgTCsBVYlg8fosOkaEcZ11EhvToJycHDp27EhCQgIi4u9wgpqqUlhYSE5ODomJiV4fZ01D4DQLxV4EnftQdKqCf2w/yA2jehPdzvKkMQ0pLS2lW7dulgRaARGhW7dujb46s0TgKoP9a6umlXhnSw5lLreNHTCmESwJtB5N+W9hiSB7gzPt9IApnk7ibEbGd2Z4n84NHmqMMW2BJYL0VJBQSJjEluxjfJN3wqabNsYEFUsE6akQnwyRnVm8PovodqFcn9Tb31EZY1ohl8vl7xB8Irh7Q0uOwoHNcPlPOV5awfvbDnDTqD50iAjur8WYpvrV+zvYeeB4s55zaO9OPHn9sAbL3XjjjWRnZ1NaWsoDDzzAfffdx0cffcTjjz9OZWUlsbGxfPrppxQXF3P//feTlpaGiPDkk08yY8YMOnToQHFxMQBLly7lgw8+4OWXX+bOO+8kMjKSLVu2MGnSJObMmcMDDzxAaWkpUVFRvPTSSwwaNIjKykoeeeQRPvroI0JCQrj33nsZNmwYzz//PO+++y4An3zyCX/605945513mvU7Ol/BXeNlrq6advq9rQcorXBbs5AxAWrhwoV07dqVkpISxo4dyw033MC9997LZ599RmJiIkeOHAHg17/+NZ07d2b79u0AHD16tMFz5+TksHbtWkJDQzl+/Diff/45YWFhrFixgscff5xly5axYMECMjMz2bp1K2FhYRw5coSYmBh++MMfUlBQQFxcHC+99BJ33XWXT7+HpgjuRJCeCu06oH2Sef3d9Qzt1YkR1klsTJN588vdV55//vmqX9rZ2dksWLCAyy+/vOp++q5duwKwYsUKlixZUnVcTExMg+eeOXMmoaHOnGNFRUV873vfY8+ePYgIFRUVVef9/ve/T1hYWI3Pu+OOO3j11VeZN28e69atY9GiRc30Fzef4E4E+1Kg/yS2HTzFroPH+fWNw+02OGMCUGpqKitWrGDdunVER0czZcoULr74Yr7++muvz1H93/7Z9+G3b9++6vUvf/lLrrjiCt555x0yMzOZMmVKveedN28e119/PZGRkcycObMqUbQmwdtZfCwLjuyDAVNYsjGLqPBQbrjYOomNCURFRUXExMQQHR3N119/zRdffEFpaSmfffYZGRkZAFVNQ1dffTUvvPBC1bGnm4Z69OjBrl27cLvd9bbhFxUV0aePs+ruyy+/XLX/6quv5i9/+UtVh/Lpz+vduze9e/fmmWeeYd68ec33Rzej4E0E6asAONX3ct7beoDrRvaiU6T3c3MYY1qPadOm4XK5GDJkCI8++igTJkwgLi6OBQsWcPPNN5OUlMTs2bMB+MUvfsHRo0cZPnw4SUlJpKQ4U8z89re/5brrruOSSy6hV6+6p5f52c9+xmOPPcaoUaNq3EV0zz330K9fP0aOHElSUhKvv/561Xu33XYbffv2ZciQIT76Bs6PqKq/Y2iU5ORkTUtrhkXMlt4FmatZfNknPPbOV7z9w0sY3a/htkJjTE27du1qtRVcazF//nxGjRrF3Xff3SKfV9t/ExHZpKq1LgXc+hqrWoLb7VwRXDiVxRuzGdSjI6P6dvF3VMaYNmjMmDG0b9+eZ5991t+h1Ck4E0HeV3DqMDkx49m2oYinrh9qncTGGJ/YtGmTv0NokE/7CERkmoh8IyJ7ReTRWt7/vohsF5GtIrJaRIb6Mp4qnmUpXytIJCIshJtGxbfIxxpjTGvks0QgIqHAC8A1wFBgbi0V/euqOkJVLwZ+B/zBV/HUkJ6Ku9tF/H1HBdeO6EXnaOskNsYEL19eEYwD9qpquqqWA0uAG6oXUNXqY9HbA77vua4ohf1r2dtxLMVlLuaOt5HExpjg5ss+gj5AdrXtHGD82YVE5EfAw0A74MraTiQi9wH3AfTrd54Vd84GcJXw1tELubB7B5L7251Cxpjg5vdxBKr6gqpeADwC/KKOMgtUNVlVk+Pi4s7vA9NTUQnl9bx+zBnb1zqJjTFBz5eJIBfoW2073rOvLkuAG30Yj2NfCtnRw6gIbc+M0dZJbEyw6dChg79DaHV82TS0ERgoIok4CWAOcGv1AiIyUFX3eDavBfbgSyVH0QNb+AczmDa8JzHt2/n044wJOv98FA5tb95z9hwB1/y2ec/ZCrhcrlYz75DPrghU1QXMBz4GdgFvquoOEXlaRKZ7is0XkR0ishWnn+B7vooHgIzPEZQVZcNsumlj2ohHH320xtxBTz31FM888wxTp05l9OjRjBgxgvfee8+rcxUXF9d53KJFi6qmj7jjjjsAyMvL46abbiIpKYmkpCTWrl1LZmYmw4cPrzru97//PU899RQAU6ZM4cEHHyQ5OZnnnnuO999/n/HjxzNq1Ciuuuoq8vLyquKYN28eI0aMYOTIkSxbtoyFCxfy4IMPVp33xRdf5KGHHmry91aDqgbUY8yYMdpk7z+op57qoVN/94m63e6mn8cYU2Xnzp1+/fzNmzfr5ZdfXrU9ZMgQzcrK0qKiIlVVLSgo0AsuuKDq33z79u3rPFdFRUWtx3311Vc6cOBALSgoUFXVwsJCVVWdNWuW/vGPf1RVVZfLpceOHdOMjAwdNmxY1Tn/67/+S5988klVVZ08ebL+4Ac/qHrvyJEjVXG9+OKL+vDDD6uq6s9+9jN94IEHapQ7ceKEDhgwQMvLy1VVdeLEibpt27Za/47a/psAaVpHvdo6rktaSPmelaxxDWbmuETrJDamjRg1ahT5+fkcOHCAgoICYmJi6NmzJw899BCfffYZISEh5ObmkpeXR8+ePes9l6ry+OOPn3PcypUrmTlzJrGxscCZtQZWrlxZtb5AaGgonTt3bnChm9OT34Gz4M3s2bM5ePAg5eXlVWsn1LVmwpVXXskHH3zAkCFDqKioYMSIEY38tmoXPIng6H7aFWWyjsn8YIx1EhvTlsycOZOlS5dy6NAhZs+ezWuvvUZBQQGbNm0iPDychISEc9YYqE1Tj6suLCwMt9tdtV3f2gb3338/Dz/8MNOnTyc1NbWqCaku99xzD7/5zW8YPHhws05p7ffbR1tKxZ6VzovEKcR2iPBvMMaYZjV79myWLFnC0qVLmTlzJkVFRXTv3p3w8HBSUlLYv3+/V+ep67grr7ySt956i8LCQuDMWgNTp07lz3/+MwCVlZUUFRXRo0cP8vPzKSwspKysjA8++KDezzu9tsErr7xStb+uNRPGjx9PdnY2r7/+OnPnzvX262lQ0CSCjUejWVZ5GVdcepm/QzHGNLNhw4Zx4sQJ+vTpQ69evbjttttIS0tjxIgRLFq0iMGDB3t1nrqOGzZsGD//+c+ZPHkySUlJPPzwwwA899xzpKSkMGLECMaMGcPOnTsJDw/niSeeYNy4cVx99dX1fvZTTz3FzJkzGTNmTFWzE9S9ZgLArFmzmDRpkldLbHoraNYjWLEzjzfSsvnL7WMICbH+AWOai61H0LKuu+46HnroIaZOnVpnmcauRxA0VwRXDe3Bi99NtiRgjAlIx44d46KLLiIqKqreJNAUwdNZbIwxHtu3b68aC3BaREQE69ev91NEDevSpQu7d+/2ybktERhjzpuqBtQt2SNGjGDr1q3+DsMnmtLcHzRNQ8YY34iMjKSwsLBJFZBpXqpKYWEhkZGRjTrOrgiMMeclPj6enJwcCgoK/B2KwUnM8fGNGytlicAYc17Cw8OrRsSawGRNQ8YYE+QsERhjTJCzRGCMMUEu4EYWi0gB4N3EIeeKBQ43YziBzr6Pmuz7OMO+i5rawvfRX1VrXes34BLB+RCRtLqGWAcj+z5qsu/jDPsuamrr34c1DRljTJCzRGCMMUEu2BLBAn8H0MrY91GTfR9n2HdRU5v+PoKqj8AYY8y5gu2KwBhjzFksERhjTJALmkQgItNE5BsR2Ssij/o7Hn8Rkb4ikiIiO0Vkh4g84O+YWgMRCRWRLSJS9wKzQUJEuojIUhH5WkR2ichEf8fkLyLykOffyVcislhEGjetZ4AIikQgIqHAC8A1wFBgrogM9W9UfuMCfqKqQ4EJwI+C+Luo7gFgl7+DaCWeAz5S1cFAEkH6vYhIH+DHQLKqDgdCgTn+jco3giIRAOOAvaqarqrlwBLgBj/H5BeqelBVN3ten8D5R97Hv1H5l4jEA9cCf/V3LP4mIp2By4G/Aahquaoe829UfhUGRIlIGBANHPBzPD4RLImgD5BdbTuHIK/8AEQkARgFtN71+VrGfwM/A9z+DqQVSAQKgJc8TWV/FZH2/g7KH1Q1F/g9kAUcBIpU9V/+jco3giURmLOISAdgGfCgqh73dzz+IiLXAfmqusnfsbQSYcBo4M+qOgo4CQRln5qIxOC0HCQCvYH2InK7f6PyjWBJBLlA32rb8Z59QUlEwnGSwGuq+ra/4/GzScB0EcnEaTK8UkRe9W9IfpUD5Kjq6avEpTiJIRhdBWSoaoGqVgBvA5f4OSafCJZEsBEYKCKJItIOp8NnuZ9j8gtxVhj/G7BLVf/g73j8TVUfU9V4VU3A+f9ipaq2yV993lDVQ0C2iAzy7JoK7PRjSP6UBUwQkWjPv5uptNGO86BYqlJVXSIyH/gYp+d/oaru8HNY/jIJuAPYLiJbPfseV9UP/RiTaV3uB17z/GhKB+b5OR6/UNX1IrIU2Ixzt90W2uhUEzbFhDHGBLlgaRoyxhhTB0sExhgT5CwRGGNMkLNEYIwxQc4SgTHGBDlLBMZ4iEiliGyt9mi2EbUikiAiXzXX+YxpTkExjsAYL5Wo6sX+DsKYlmZXBMY0QEQyReR3IrJdRDaIyIWe/QkislJEtonIpyLSz7O/h4i8IyJfeh6npyUIFZEXPfPb/0tEojzlf+xZH2KbiCzx059pgpglAmPOiDqraWh2tfeKVHUE8L84s5UC/A/wiqqOBF4Dnvfsfx5YpapJOPP0nB7FPhB4QVWHAceAGZ79jwKjPOf5vq/+OGPqYiOLjfEQkWJV7VDL/kzgSlVN90zYd0hVu4nIYaCXqlZ49h9U1VgRKQDiVbWs2jkSgE9UdaBn+xEgXFWfEZGPgGLgXeBdVS328Z9qTA12RWCMd7SO141RVu11JWf66K7FWUFvNLDRswiKMS3GEoEx3pld7Xmd5/VazixdeBvwuef1p8APoGot5M51nVREQoC+qpoCPAJ0Bs65KjHGl+yXhzFnRFWbkRWcdXtP30IaIyLbcH7Vz/Xsux9nJa+f4qzqdXqWzgeABSJyN84v/x/grHBVm1DgVU+yEOD5IF8a0viB9REY0wBPH0Gyqh72dyzG+II1DRljTJCzKwJjjAlydkVgjDFBzhKBMcYEOUsExhgT5CwRGGNMkLNEYIwxQe7/A5Kej4eapTd3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3wVdbr48c9zThohhRIgkEaQDpEWigWsV7EsWBcRRF3rqqi7Xq/u1d3ruu69e9ff3aLiuqyra0GQBQtiwVVALLQQQhcILSShJLRAMPV8f3/MCZyElHNCJpPkPO/XnteZM/OdOU/O4jwz8535PmKMQSmlVPByOR2AUkopZ2kiUEqpIKeJQCmlgpwmAqWUCnKaCJRSKsiFOB1AoOLi4kzPnj2dDkMppVqVNWvWFBpjutS2rNUlgp49e5KRkeF0GEop1aqIyJ66lumlIaWUCnKaCJRSKshpIlBKqSDX6voIlFLBqby8nNzcXEpKSpwOpUWLiIggMTGR0NBQv9fRRKCUahVyc3OJjo6mZ8+eiIjT4bRIxhgOHTpEbm4uqampfq+nl4aUUq1CSUkJnTt31iRQDxGhc+fOAZ81aSJQSrUamgQa1pjfKGgSwa7CYn790SbKKz1Oh6KUUi1K0CSCnQUneP3b3XywNs/pUJRSrVRUVJTTIdjCtkQgIq+JyEER2VhPm4tFJEtENonIV3bFAnBp/64M6hHDy0t3UKFnBUopdYqdZwT/AMbXtVBEOgAvAxOMMYOAm22MBRFh+qV92FVYzML1++z8KqVUG2eM4fHHH2fw4MGkpaXx7rvvArBv3z7GjRvH0KFDGTx4MF9//TWVlZXccccdp9r+8Y9/dDj6M9l2+6gxZpmI9Kynya3Ae8aYHG/7g3bFUuWKgd3o1y2al5ZkM2FID1wu7XhSqjX69Ueb2Jxf1KTbHNgjhv/60SC/2r733ntkZWWxbt06CgsLGTlyJOPGjeOdd97hyiuv5KmnnqKyspKTJ0+SlZVFXl4eGzdaF0eOHj3apHE3BSf7CPoCHUVkqYisEZFpdTUUkXtFJENEMgoKChr9hS6X8NClvck+eIJPN+5v9HaUUsHtm2++YfLkybjdbrp168ZFF13E6tWrGTlyJK+//jrPPPMMGzZsIDo6ml69erFz506mT5/OZ599RkxMjNPhn8HJB8pCgBHAZUA7YLmIrDDGbKvZ0BgzE5gJkJ6ebs7mS69O684fv9jGi4u3c9XgeD0rUKoV8vfIvbmNGzeOZcuW8fHHH3PHHXfw85//nGnTprFu3ToWLVrEK6+8wty5c3nttdecDrUaJ88IcoFFxphiY0whsAwYYveXul3CQ5f05vv9x/liywG7v04p1QaNHTuWd999l8rKSgoKCli2bBmjRo1iz549dOvWjXvuuYe7776bzMxMCgsL8Xg83HjjjTz33HNkZmY6Hf4ZnDwj+BB4SURCgDBgNNAsvSgThvTgz19u58XF2fzbwG76kIpSKiDXX389y5cvZ8iQIYgIv//974mPj+eNN97g+eefJzQ0lKioKN58803y8vK488478XisuxX/53/+x+HozyTGnNWVlro3LDIbuBiIAw4A/wWEAhhjXvG2eRy4E/AArxpj/tTQdtPT001TFKZ5d3UOT8zfwOt3juSSfl3PentKKXtt2bKFAQMGOB1Gq1DbbyUia4wx6bW1t/Ouocl+tHkeeN6uGOpz/bBEXvgymxe+3M7FfbvoWYFSKmgFzZPFNYWFuPjpxeewNuco32YfcjocpZRyTNAmAoCb0xOJj4nghcXbnQ5FKaUcE9SJIDzEzX0X9WLVrsOs3KlnBUqp4BTUiQBg8qhk4qLCeXFxttOhKKWUI4I+EUSEurl3XCrfZBeSmXPE6XCUUqrZBX0iAJgyOoWOkaG8+KX2FSilgo8mAqB9eAh3j+3Fkq0FrM9teQNCKaVan/pqF+zevZvBgwc3YzT100TgNe28FGIiQrSvQCkVdJwcYqJ5VVbA7q+h18VQy8Nj0RGh/OTCVP70xXa27CtiQPeWN0KgUsrr0ydh/4am3WZ8Glz1uzoXP/nkkyQlJfHggw8C8MwzzxASEsKSJUs4cuQI5eXlPPfcc0ycODGgry0pKeGnP/0pGRkZhISE8Ic//IFLLrmETZs2ceedd1JWVobH42H+/Pn06NGDH//4x+Tm5lJZWckvf/lLJk2adFZ/NgTTGcG62fDWdZBf94BPd56fSlR4CC/pWYFSqoZJkyYxd+7cU5/nzp3L7bffzvvvv09mZiZLlizhscceI9Bhe2bMmIGIsGHDBmbPns3tt99OSUkJr7zyCo888ghZWVlkZGSQmJjIZ599Ro8ePVi3bh0bN25k/Pg6a38FJHjOCAZOhE//AzLfgoQRtTaJjQzl9vNTeHnpDrIPHqd31+hmDlIp5Zd6jtztMmzYMA4ePEh+fj4FBQV07NiR+Ph4fvazn7Fs2TJcLhd5eXkcOHCA+Ph4v7f7zTffMH36dAD69+9PSkoK27Zt47zzzuO3v/0tubm53HDDDfTp04e0tDQee+wxnnjiCa699lrGjh3bJH9b8JwRRMTAwOtgwzwoK66z2V0X9qJdqFvPCpRSZ7j55puZN28e7777LpMmTWLWrFkUFBSwZs0asrKy6NatGyUlJU3yXbfeeisLFiygXbt2XH311SxevJi+ffuSmZlJWloaTz/9NM8++2yTfFfwJAKA4dOg7Dhs/rDOJp3ahzF1TAoL1uWzq7DuhKGUCj6TJk1izpw5zJs3j5tvvpljx47RtWtXQkNDWbJkCXv27Al4m2PHjmXWrFkAbNu2jZycHPr168fOnTvp1asXDz/8MBMnTmT9+vXk5+cTGRnJ1KlTefzxx5ustkFwJYLkMdC5D2S+WW+zu8emEup2MWOJnhUopU4bNGgQx48fJyEhge7duzNlyhQyMjJIS0vjzTffpH///gFv84EHHsDj8ZCWlsakSZP4xz/+QXh4OHPnzmXw4MEMHTqUjRs3Mm3aNDZs2MCoUaMYOnQov/71r3n66aeb5O+yrR6BXc66HsG3f4Z//QoeXA1d+tbZ7JkFm3hrxR6W/vvFJHWKbPz3KaWahNYj8F+g9QhsOyMQkddE5KCIbGyg3UgRqRCRm+yKpZohk8EVAmvfqrfZ/Redg1uEl5fuaJawlFLKKXZeGvoHUO+9TSLiBv4X+NzGOKqL6gp9x1u3k1aW19ksPjaCH49MZN6aveQf/aHZwlNKtR0bNmxg6NCh1V6jR492Oqwz2JYIjDHLgMMNNJsOzAcO2hVHrYbfDsUFsO2zepvdf9E5GAN//UrPCpRqCVrbpey0tDSysrKqvVauXGnrdzbmN3Kss1hEEoDrgb/40fZeEckQkYyCgoKz//Lel0F0jwY7jRM7RnLj8ERmr97LwaKmuSVMKdU4ERERHDp0qNUlg+ZkjOHQoUNEREQEtJ6TD5T9CXjCGONpqF6wMWYmMBOszuKz/maXG4ZNga//D47lQWxCnU0fuOQc5mXmMnPZTp6+duBZf7VSqnESExPJzc2lSQ4G27CIiAgSExMDWsfJRJAOzPEmgTjgahGpMMZ80CzfPnQKLHsest6Bix6vs1lK5/ZMHNKDWStzuP/ic4iLCm+W8JRS1YWGhpKamup0GG2SY5eGjDGpxpiexpiewDzggWZLAgCdUiH1Ilj7Jng89TZ94JLelFRU8urXu5opOKWUaj523j46G1gO9BORXBG5S0TuF5H77frOgA2fBkdzYPeyepv17hrFNWndeWv5bo4UlzVPbEop1UxsuzRkjJkcQNs77IqjXv2vhYgOVqdxr4vrbTr90j4sXL+P17/dxc+v6Ncs4SmlVHMIriEmagqNgCG3wJaP4GT9d7r2i49m/KB4Xv9uN0UldT9/oJRSrU1wJwKAYbdBZRmsn9tg04cu7c3xkgre+Ha3/XEppVQz0UQQPxh6DLcuDzVwf/LghFgu69+Vv3+7ixOlFc0UoFJK2UsTAcDw2+Dgpnqrl1WZflkfjp4s5+0VgQ83q5RSLZEmAoDBN0FoZINPGgMMTerA2D5x/G3ZTk6W6VmBUqr100QAVvWyQdfDhvn1Vi+r8vBlfThUXMY7K3OaITillLKXJoIqw26zqpdtaviZtpE9O3Fer87MXLaTkvLKZghOKaXso4mgSlX1sgbqFFSZfllvDh4vZW7GXpsDU0ope2kiqCJidRrnLIeCbQ02P69XZ9JTOvLK0h2UVdQ/RIVSSrVkmgh8+Vm9DEBEmH5ZH/KPlTA/M7cZglNKKXtoIvDlW72souExhcb1iWNIYiwvL82mvFLPCpRSrZMmgpr8rF4G3rOCS/uw9/APfLA2rxmCU0qppqeJoKaq6mV+dhpfNqArA7vH8PLSHVR6tHKSUqr10URQU1X1suwvrOplDbDOCnqzq7CYhevzmyFApZRqWpoIajNsKhiPVb3MD1cOiqdvtyheWpyNR88KlFKtjJ2FaV4TkYMisrGO5VNEZL2IbBCR70RkiF2xBKxjT7+rlwG4XMJDl/Zh+8ETfLZpv/3xKaVUE7LzjOAfwPh6lu8CLjLGpAG/wVucvsXws3pZlWvSutOrS3teXJyNaWAUU6WUaklsSwTGmGVAndVejDHfGWOOeD+uABLtiqVR+l8L7Tr6NRAdgNslPHhxb7bsK+KLLQdtDk4ppZpOS+kjuAv41OkgqgmNgHMn+VW9rMrEoT1I7hTJC19u17MCpVSr4XgiEJFLsBLBE/W0uVdEMkQko6CgoPmCC6B6GUCI28UDF5/DhrxjLN3WjHEqpdRZcDQRiMi5wKvARGPMobraGWNmGmPSjTHpXbp0ab4AA6heVuWG4YkkdGjHi3pWoJRqJRxLBCKSDLwH3GaMaXiUN6cMn+Z39TKAsBAX91/Ui8yco3y3o87cppRSLYadt4/OBpYD/UQkV0TuEpH7ReR+b5NfAZ2Bl0UkS0Qy7IrlrAy+0e/qZVVuTk+iW0w4L3y53cbAlFKqaYTYtWFjzOQGlt8N3G3X9zcZ3+plV/43hLVveJVQN/eNO4dnF25m1a7DjErt1AyBKqVU4zjeWdwqDJ/md/WyKpNHJRMXFcaLi/WsQCnVsmki8EfSaKt6WQCXh9qFublnbC++3l5IZs6RhldQSimHaCLwh4h1VrB3hV/Vy6pMHZNCx8hQXtS+AqVUC6aJwF+nqpf5f1bQPjyEuy5MZcnWAjbkHrMxOKWUajxNBP6K6gL9roJ1c/yqXlZl2vk9iYkI0b4CpVSLpYkgEMOm+V29rEpMRCh3XJDK55sPsGVfkY3BKaVU42giCERV9bIAOo0BfnJBT6LCQ3hpSbZNgSmlVONpIgiEy20VrdnxJRzL9Xu1DpFhTDsvhU827CP74HEbA1RKqcBpIgjUsCkBVS+rcteFqUSEuJmxZIdNgSmlVONoIghUx57Q62KruL0f1cuqdI4KZ+qYZD7MymNXYbFd0SmlVMA0ETTGsNus6mW7vgpotXvG9SLU7eJl7StQSrUgmggao6p62dq3Alqta3QEk0cl8/7aPA4WldgUnFJKBUYTQWM0onpZldvP70mFx/Du6r02BaeUUoHRRNBYp6qXvRvQaqlx7bmwdxyzV+VQ6dHCNUop52kiaKz4wZAwIqDqZVWmjE4m/1gJS77XIvdKKedpIjgbw26Dg5shz7/qZVUuH9iNrtHhvL1yj02BKaWU/+ysUPaaiBwUkY11LBcReUFEskVkvYgMtysW21RVLwtgIDqAULeLW0Yl89W2AvYePmlTcEop5R87zwj+AYyvZ/lVQB/v617gLzbGYg/f6mVlgT0bMHlUEgLMWpljT2xKKeUn2xKBMWYZUN8tNROBN41lBdBBRLrbFY9tGlG9DKB7bDsuG9CNf2bspbSi0qbglFKqYU72ESQAvvdQ5nrnnUFE7hWRDBHJKCgoaJbg/NaI6mVVpo5J4VBxGZ9t3G9DYEop5Z9W0VlsjJlpjEk3xqR36dLF6XCqq1a9bGtAq47tHUdK50hmrdDLQ0op5ziZCPKAJJ/Pid55rc+p6mWBPWnscgm3jkpm1e7DbN2vo5IqpZzhZCJYAEzz3j00BjhmjNnnYDyNV1W9LGt2QNXLAG4akUiY28U7eiupUsohdt4+OhtYDvQTkVwRuUtE7heR+71NPgF2AtnA34AH7IqlWQy/HU4WBlS9DKxRSa9Oi+e9zDyKSytsCk4ppeoWYteGjTGTG1hugAft+v5md86lEJNgdRoPnBDQqlPHpPBBVj4L1uUzeVSyTQEqpVTtWkVncavgcsPQKQFXLwMYkdKR/vHRvL1iDybA4SqUUupsaSJoSo2sXiYiTBmdzKb8ItblHrMpOKWUqp0mgqbUyOplANcNSyAyzM3bK7TTWCnVvDQRNLXh0xpVvSw6IpTrhiXw0bp8jp4M7M4jpZQ6G5oImlpV9bLGPGk8OoXSCg/z1gTWx6CUUmdDE0FTCwmHc2+B7xcGXL1sYI8Yhid34J2VOdpprJRqNpoI7DC8cdXLAKaMTmFnYTHLdxyyITCllDqTJgI7dBvU6Opl15zbnQ6RoVq0RinVbDQR2KWR1csiQt3cPCKRzzcd4GBRiU3BKaXUaZoI7FJVvSzzjYBXvXV0ChUew7ur9zbcWCmlzpJfiUBEHhGRGO8AcX8XkUwRucLu4Fq1iBgYdANsnA+lJwJaNTWuPRf2jmP2qhwqPdpprJSyl79nBD8xxhQBVwAdgduA39kWVVsx/DYoOwGbA6teBjB1TDL5x0pY/P1BGwJTSqnT/E0E4n2/GnjLGLPJZ56qS9JoiOsLmYHVKQC4fEA3usWE65PGSinb+ZsI1ojI51iJYJGIRAOBjaEQjESsTuNGVC8Lcbu4ZWQyy7YXkHPopE0BKqWU/4ngLuBJYKQx5iQQCtxpW1RtSSOrlwHcMioJlwjvrNJSlkop+/ibCM4DthpjjorIVOBpoMFhMkVkvIhsFZFsEXmyluXJIrJERNaKyHoRuTqw8FuBs6he1j22HZf178rcjL2UVlTaFKBSKtj5mwj+ApwUkSHAY8AOoN7BdETEDcwArgIGApNFZGCNZk8Dc40xw4BbgJcDiL31OFW97NOAV506JoXDxWV8tnG/DYEppZT/iaDCW1FsIvCSMWYGEN3AOqOAbGPMTmNMGTDHu74vA8R4p2OBfD/jaV1OVS8L/PLQhb3jSOkcyawVenlIKWUPfxPBcRH5BdZtox+LiAurn6A+CYDvE1G53nm+ngGmikguVg3j6bVtSETuFZEMEckoKCjwM+QWpKp6WfYXAVcvc7mEW0cls2r3YbbuP25TgEqpYOZvIpgElGI9T7AfSASeb4Lvnwz8wxiTiPfWVG+SqcYYM9MYk26MSe/SpUsTfK0Dhk213gOsXgZwc3oSYSEuZun4Q0opG/iVCLw7/1lArIhcC5QYYxoacD8PSPL5nOid5+suYK73O5YDEUCcPzG1Oh1ToNdF1uWhAKuXdWofxjVp3XkvM4/i0gqbAlRKBSt/h5j4MbAKuBn4MbBSRG5qYLXVQB8RSRWRMKzO4AU12uQAl3m/YwBWImiF1378NHwaHMuBXUsDXnXqmGROlFawYF3b7EZRSjnH30tDT2E9Q3C7MWYaVkfwL+tbwRhTATwELAK2YN0dtElEnhWRCd5mjwH3iMg6YDZwh2nLFVlOVS8LvNN4eHJH+sdH8/aKPVq0RinVpEL8bOcyxvgOenMIP5KIMeYTrE5g33m/8pneDFzgZwytX0i49YDZqpmwfwPEp/m9qogwZUwKv/xgI1l7jzIsuaONgSqlgom/ZwSficgiEblDRO4APqbGDl75aey/Q2RneO8+qCgNaNXrhyXQPszN23orqVKqCfnbWfw4MBM41/uaaYx5ws7A2qz2nWHCi3BwEyz574BWjQoP4bphCSxcn8/Rk4E9payUUnXxuzCNMWa+Mebn3tf7dgbV5vW90nra+Ns/w57lAa06ZXQKpRUe5q0J7HkEpZSqS72JQESOi0hRLa/jIlLUXEG2SVf+Fjokwwf3B1S4ZmCPGIYnd+CdlTnaaayUahL1JgJjTLQxJqaWV7QxJqa+dVUDwqPh+lfgyB74/OmAVp06JoWdhcV8t+OQTcEppYKJ1ix2Usr5cMHDsOZ12Pa536tdndadDpGh+qSxUqpJaCJw2iVPQdeBsOAhOHnYr1UiQt3cPCKRzzcd4GBRic0BKqXaOk0ETgsJh+v/aiWBhT8DP6/73zo6hQqPYc7qvQ03VkqpemgiaAm6nwuX/MIqcr9xvl+rpMa1Z2yfOGavyqGiUquGKqUaTxNBS3H+I5A4Cj7+ORT5N57QlNEp7DtWwpKtbXd4JqWU/TQRtBTuEOsuospy+PBBvy4RXT6gK91iwnl7hXYaK6UaTxNBS9L5HLjiN7BjMWT8vcHmIW4Xt4xMZtn2AnIOnWyGAJVSbZEmgpYm/S445zL4/JdwaEeDzSePSsYlwqxVelaglGocTQQtjQhMfAncofD+/VBZfyGa+NgILh/QlX9m5FJaUdlMQSql2hJNBC1RTA+45g+Quwq+/VODzaeMTuFwcRmfbdzfDMEppdoaWxOBiIwXka0iki0iT9bR5scisllENolI4AV926q0m2DQDbD0f2DfunqbXtg7jpTOkdpprJRqFNsSgYi4gRnAVcBAYLKIDKzRpg/wC+ACY8wg4FG74mmVrvk/iIyzLhGV1/0EscslTBmdzOrdR/h+v44FqJQKjJ1nBKOAbGPMTmNMGTAHmFijzT3ADGPMEYAaVdBUZCerv+DgZljy23qb3jQiibAQF++s1KI1SqnA2JkIEgDf8Q9yvfN89QX6isi3IrJCRMbXtiERuVdEMkQko6AgyB6e6vNvMOJO+O5F2PNdnc06tQ/jmrTuvJeZR3Fp/R3MSinly+nO4hCgD3AxMBn4m4h0qNnIGDPTGJNujEnv0qVLM4fYAlzxHHRMsS4RlR6vs9nUMcmcKK3gwyz/nkxWSimwNxHkAUk+nxO983zlAguMMeXGmF3ANqzEoHyFR1kD0x3NgUVP1dlseHJH+sdH8/aKPVq0RinlNzsTwWqgj4ikikgYcAuwoEabD7DOBhCROKxLRTttjKn1Sh4DFzwCmW/AtkW1NhERpo5JYfO+IrL2Hm3mAJVSrZVticAYUwE8BCwCtgBzjTGbRORZEZngbbYIOCQim4ElwOPGGC27VZdL/hO6DoIPH4Li2n+m64Yl0D7MzdsrtNNYKeUfW/sIjDGfGGP6GmPOMcb81jvvV8aYBd5pY4z5uTFmoDEmzRgzx854Wr2QcLjhr/DDEfi49toFUeEhXDcsgYXr8zl6ssyBIJVSrY3TncUqUPFpcOlTsPlD2PDPWptMHZNCaYWHeWtymzk4pVRrpImgNTr/YUgaDR//Oxyr2f8OA7rHMCKlI7NW5minsVKqQZoIWiOX26pd4KmADx8Az5kVyqaOSWZXYTHf7dAuF6VU/TQRtFadesGVz8HOpbXWLrhqcHc6Robq+ENKqQZpImjNRtwJvS+3ahcUZldbFBHq5ub0JD7ffIADRXWPU6SUUpoIWjMRmPCSdTfR+/edUbvg1lHJVHoM767eW8cGlFJKE0HrF9Mdrv0D5GXAt3+stqhnXHvG9olj9qocKirP7EdQSinQRNA2DL7Rei393Rm1C6aMTmHfsRIWf68DuyqlaqeJoK24+v9ZtQveu69a7YLLB3QlPiaCt3V4aqVUHTQRtBWRnWDiDCjYAot/c2p2iNvFLaOSWLatgJxDJx0MUCnVUmkiaEv6XA7pd8HyGbD7m1OzbxmZjNslzFqlt5Iqpc6kiaCtueI30LEnfPBTKLHKVsbHRnD5gK78MyOX0opKZ+NTSrU4mgjamrD2Vu2CY7mw6D9PzZ46JoXDxWV8umG/g8EppVoiTQRtUfJouOBRWPsWbP0UgAvOiaNn50hmrdTLQ0qp6jQRtFUX/wK6pcGC6VBciMslTBmdwurdR/h+f5HT0SmlWhBbE4GIjBeRrSKSLSJP1tPuRhExIpJuZzxBJSTMGpiu5BgsfBSM4aYRiYSFuJilRWuUUj5sSwQi4gZmAFcBA4HJIjKwlnbRwCPASrtiCVrxg+GSp2DLR7B+Lh3bh3FtWnfeX5tHcWlFw+srpYKCnWcEo4BsY8xOY0wZMAeYWEu73wD/C+jIaHY4fzokjYFPHodjuUwZk8KJ0go+zMp3OjKlVAthZyJIAHxHO8v1zjtFRIYDScaYj22MI7i53HD9X6zaBR88wPCkGAZ0j+HtFXu0aI1SCnCws1hEXMAfgMf8aHuviGSISEZBQYH9wbU1nXrBlb+FXV8hq19lyuhkNu8rYu3eo05HppRqAexMBHlAks/nRO+8KtHAYGCpiOwGxgALauswNsbMNMakG2PSu3TpYmPIbdiIO6DPFfCvX3F9cjHtw9z89asdVHr0rECpYGdnIlgN9BGRVBEJA24BFlQtNMYcM8bEGWN6GmN6AiuACcaYDBtjCl4iMOFFCG1H+4UPct/YFBZtOsB9b2VwQjuOlQpqtiUCY0wF8BCwCNgCzDXGbBKRZ0Vkgl3fq+oRHQ/X/hHyM5keuoBfTxjEkq0F3PSX79h7WAekUypYSWvrMExPTzcZGXrScFbm3w2b3oe7v+Dr4kQenJVJiNvFK1NHMCq1k9PRKaVsICJrjDG1PqulTxYHo6ufh/Zd4LXxjF3zKF9cvo/EiFKmvLqCuVrWUqmgE+J0AMoB7TrCtAWw+m+wZSFdv1/Ih64QNkQPYc4HQ8nLvZ6HJ16I2yVOR9o8jIGjOdals5Bwp6NRqtnppaFg5/FAfiZsWYDZvAA5sguPEbZHDCLlgklEpE2EjilOR9n0juXCzq9g11fW+4n9EB4DfcfDgB9B78shLNLpKJVqMvVdGtJEoE4zBg5uZv2/3iZ020IGuLxjEnUfYu0cB0yALv2cjbGxfjgCu7727viXwqFsa35kHKSOg+QxsG89bP3YahvSzir0M2Ai9L0CImIdDV+ps6WJQAVs+Y5D/PfbC7mUVdzdeRPRhWutBXF9vUnhR9B9qHVbaktU/gPkrDi949+3DowHQttDzwsg9SLodTF0HQgun66yygrY8401PtOWhdaZgmStGe0AABGkSURBVDvMajvgR9DvGmjf2ZE/SamzoYlANcqeQ8Xc/UYGuwqL+d8r4rgxch1sWQC7vwVTCbFJp5NC0mhrOAuneCohPwt2LrF2/jkrobIUXCGQOPL0jj9hhDUyq1/b9EDuautv3rLA6kcQF6RcAAMnQv9rIKaHnX+VUk1GE4FqtOMl5Tw8ey1LthYw7bwUfnXtQEJKjsC2T62j5h2LobLMugup/zVWUug5zv+dbWMZA4XbraP9XV9Zl31Kj1nLug22dvqpF0HKeRAe3TTft2+d90xhARRus+YnjrL+5oETrBKhSrVQmgjUWan0GH736Rb+9vUuLuwdx4xbhxMbGWotLCmC7H9ZO8htn0N5MYTHQj9vp+s5lzVdp2tRvk8H71I4vs+a3yH59I4/9SKIaoZhSAq2wmbvmcL+9da8+HOtfpSBrbgvRdnPGKgogbJiKDvhfa9v2udz3/GQdlOjvlYTgWoSczP28tT7G0jsGMnfpqXTu2tU9QblP1g76C0fwdZPanS6TrDGOmrXwf8v/OEo7P7m9I6/6ii8XSfoddHpyz2dUpvmD2ysw7u8ZwofQe4qa15cX+tvHvAjq7O9pfalqIaV/2D9W662Y/Zzx13bsvJiq7/KX2FRVi3ysPaQ/hNraPlG0ESgmkzG7sPc99Yayio9zLh1OOP61nH0XVkOe76t3unqCrV24FWdrjWP3MtLYO/K05d78td6O3gjIeX800f93QZX7+BtSYry4fuPYfOH1t9vPNYZy4AJ1itxZMuNPdiUHofjB6x/m8e9rxP7rXnH98GJA9Z01SXHhrhCvDttnx13WHufz5E1ltXVzmc6pF2T/XvRRKCaVO6Rk9z9RgbbDhznl9cO5I7zeyL1HfF6PJCX4e10/QiO7LY6XZPPg/7XWqfJu76y7vKpKAFxQ2L66R1/4kj7+xzsUHzIuh11y0ewYwl4yiEqHgZcayXDlAvBrc90NiljrPKsvjv1ajv6A6eny4vPXD8kAqK6WQ8XRsdb/39Fd7MewgyLrn8n3sL/jWoiUE2uuLSCR9/N4l+bDzB5VBK/njCYsBA/jlyMgQMbT19KObjZmt91oE8H7/kQEWNn+M2v5JjVh7LlQ9j+BVT8YF3i6n+1dabQ62J9qrk+xsDJw7Ufvdfc0VfUUuwwtL3Pzr2WHX2U93NEbJu9jKeJQNnC4zH837+2MmPJDkalduKVqSPo1D7Ao6Iju63T3+hutsTYIpWdhOwvrDOkbYugtMg62ux7pZUYuvSHmIQ2vVM6Q3kJHNsLR/bA0d3e9xzrCfATB6xXZdmZ64XHenfk3SC6e/Wduu+OvinuHGvlNBEoW32wNo//mL+ebjHh/P32kfTtpv/R+a2i1LoTassCq2/hh8Onl4VFQWyilRRiE6znNmISrHmxidYzDKHtnIs9EJUVUJR7egd/dI932vt+Yn/19u4wq28lNtHawZ+xo/e+6zAgftNEoGy3NucI9761hh/KKvnzLUO5bEAQHeE3lcoK61mFo3usI+GiPOu9arq4ljKtkXFWkojxJofYBG+ySLKmo+Kbpx/C47GO2mvu4I96X8fyrIcQq4jbiq9DivXqWOM9qpt2qjcxTQSqWew79gP3vJnBpvwinhzfn3vH9aq/E1kFprzESghFedaO9ViudZR9ajrPuszkS9zWkXRsgs/ZRWL16cjODV+CqrpG73vZptrOfq/1JLevqPjTO/YOydV39jEJ4A5t0p9H1c+xRCAi44E/A27gVWPM72os/zlwN1ABFAA/McbsqW+bmghath/KKvn3f67j4w37uHF4Iv99w2DCQxwceiLYlByzEkNRnnXN/dS0z5lFzWvtIRFnXn6KiLHaH805vbMvO1F9vXYdaz+a75ACHZJaz2WrIFFfIrDtnFFE3MAM4N+AXGC1iCwwxmz2abYWSDfGnBSRnwK/BybZFZOyX7swNy9OHkafblH86Yvt7D5UzCtTR9AlWu+IaRYRsdar28Dal3s8cLLQ59KTN2FUTe9YYl2vrxqgr2rnnjq2xs4+ue3d2RXE7Lx4OArINsbsBBCROcBE4FQiMMYs8Wm/AphqYzyqmbhcwqOX96VP12ge+2cW1834lr9NS2dgD91xOM7lgqiu1itheO1tKsuto/+IDsFz11KQs7M3JgHwrXuY651Xl7uAT2tbICL3ikiGiGQUFNTSYaZapGvO7c4/7zufSo/hxr98x2cb9ze8knKeO9S67KNJIGi0iG55EZkKpAPP17bcGDPTGJNujEnv0qUZBhRTTSYtMZYFD11A3/ho7n97DTOWZNPablBQqq2zMxHkAUk+nxO986oRkcuBp4AJxpjSmstV69c1JoJ37x3DdUN78PyirTwyJ4uS8sqGV1RKNQs7E8FqoI+IpIpIGHALsMC3gYgMA/6KlQQO2hiLclhEqJs/ThrK41f2Y8G6fCb9dTkHimoZCkAp1exsSwTGmArgIWARsAWYa4zZJCLPisgEb7PngSjgnyKSJSIL6ticagNEhAcv6c1fbxvB9oMnmPjSt2zI9XNkR6WUbfSBMuWIzflF3PNmBoeKS3n08r7cMDyBrtERToelVJulTxarFqnwRCmPzsnim+xC3C5hXJ84bhyRyOUDuhERqg+hKdWUNBGoFi374HHmZ+bxfmYe+4tKiI4I4dpze3DTiASGJ3fUYSqUagKaCFSrUOkxLN9xiPmZuXy2cT8/lFfSs3MkNwxP5PphCSR10pEmlWosTQSq1TlRWsGnG/bxXmYey3ceAmB0aiduHJHI1WndiQrXyl5KBUITgWrVco+c5P3MPN5bm8euwmIiQl2MHxTPjSMSOf+cONwuvXSkVEM0Eag2wRhDZs5R5mfmsnBdPkUlFcTHRHDdsARuGpFA765aEEepumgiUG1OSXklX245yPzMXL7aVkClxzAkMZYbhicyYUgPOgZaMlOpNk4TgWrTCo6X8mFWHvMz89iyr4hQt3Bp/67cMDyRS/p1JSykRQyppZSjNBGooLE5v4j3MnP5ICufwhOldIwMZcKQHtw4IpG0hFi9FVUFLU0EKuhUVHr4ensh8zJz+dfmA5RVeOjTNerUrajxsfoUswoumghUUDt2spyPN+xjfmYua/YcwSVwQe84bhyeyJWD4mkXpk8xq7ZPE4FSXrsKi3k/M5f5mXnkHf2BqPAQrk6L5/phiaQlxtI+zK2Xj1SbpIlAqRo8HsPKXYd5LzOXTzbso7jMqo8QEeqiS3Q4XaLC6RIdTpz3vbZ5Oh6Sak00EShVj5NlFSzdWsDewycpPFFKwfFSCrzvhSfKOFxcVut60REh1ZNFzXfvq1P7MELdeueSclZ9iUCf01dBLzIshKvTute5vLzSw6ETZd7EUD1RVL1vyS9i2fFSjpdWnLG+CHSMDKuWIOKiwnzONCKIi7aWd4wMw6VPSqtmZmsiEJHxwJ8BN/CqMeZ3NZaHA28CI4BDwCRjzG47Y1IqUKFuF/GxEX7daVRSXlk9UfgmD+/8PXuKOVhUSmmF54z1XQLtw0OIDg+hvfcVFR5C+3D3qemoavNDiPIuq209HX5D+cO2RCAibmAG8G9ALrBaRBYYYzb7NLsLOGKM6S0itwD/C0yyKyal7BYR6iapU2SDI6UaYzhRWnHq8pOVKEooPFHGidIKTpRWUOzzXnC8tNr8Co9/l3QjQl21JA6fBBJ2el5UxOn57UKtJOIScLkElwhuEUSwpr3LxGfaJeJti7etd9pVfdrl3Y5b5NQ6yll2nhGMArKNMTsBRGQOMBHwTQQTgWe80/OAl0RETGvruFAqQCJCdEQo0RGh9OoS2LrGGEorPBSXVlBcWllr4jjhXVZcVsHxkgpvW2v+weMlFBd61yup4IfySnv+yADUTCRViQfrf4D1m1Xd0OU7r/rnqi1Ktc9SbfrMZb7bOrUFOf0unJmsat5cVrNFbXefnTGngW3U3M4tI5O4e2yvWlqdHTsTQQKw1+dzLjC6rjbGmAoROQZ0Bgp9G4nIvcC9AMnJyXbFq1SrICJEhLqJCHXTOerst1fpMRSX+SaLSk6WVeDxgMeY069qn63pSo/BGGsbHuOd9m3jOXO60tvO4zHetlZys7ZRfdrjc0xYdXxoTn32vnvnnP5cfTmY08v8XKdqOeb0Ml81j1VrtqntUPbMNvVvo7aZcVHhtbU6a62is9gYMxOYCdZdQw6Ho1Sb4nYJMRGhxESEOh2Kcoid97TlAUk+nxO982ptIyIhQCxWp7FSSqlmYmciWA30EZFUEQkDbgEW1GizALjdO30TsFj7B5RSqnnZdmnIe83/IWAR1u2jrxljNonIs0CGMWYB8HfgLRHJBg5jJQullFLNyNY+AmPMJ8AnNeb9yme6BLjZzhiUUkrVT597V0qpIKeJQCmlgpwmAqWUCnKaCJRSKsi1umGoRaQA2NPI1eOo8dRykNPfozr9PU7T36K6tvB7pBhjah3QpNUlgrMhIhl1jccdjPT3qE5/j9P0t6iurf8eemlIKaWCnCYCpZQKcsGWCGY6HUALo79Hdfp7nKa/RXVt+vcIqj4CpZRSZwq2MwKllFI1aCJQSqkgFzSJQETGi8hWEckWkSedjsdJIpIkIktEZLOIbBKRR5yOyWki4haRtSKy0OlYnCYiHURknoh8LyJbROQ8p2Nyioj8zPvfyEYRmS0iEU7HZIegSAQi4gZmAFcBA4HJIjLQ2agcVQE8ZowZCIwBHgzy3wPgEWCL00G0EH8GPjPG9AeGEKS/i4gkAA8D6caYwVjD6bfJofKDIhEAo4BsY8xOY0wZMAeY6HBMjjHG7DPGZHqnj2P9h57gbFTOEZFE4BrgVadjcZqIxALjsGqFYIwpM8YcdTYqR4UA7bwVFCOBfIfjsUWwJIIEYK/P51yCeMfnS0R6AsOAlc5G4qg/Af8BeJwOpAVIBQqA172Xyl4VkfZOB+UEY0we8P+AHGAfcMwY87mzUdkjWBKBqoWIRAHzgUeNMUVOx+MEEbkWOGiMWeN0LC1ECDAc+IsxZhhQDARln5qIdMS6cpAK9ADai8hUZ6OyR7Akgjwgyedzonde0BKRUKwkMMsY857T8TjoAmCCiOzGumR4qYi87WxIjsoFco0xVWeI87ASQzC6HNhljCkwxpQD7wHnOxyTLYIlEawG+ohIqoiEYXX4LHA4JseIiGBdA95ijPmD0/E4yRjzC2NMojGmJ9a/i8XGmDZ51OcPY8x+YK+I9PPOugzY7GBITsoBxohIpPe/mctoox3nttYsbimMMRUi8hCwCKvn/zVjzCaHw3LSBcBtwAYRyfLO+09vjWmlpgOzvAdNO4E7HY7HEcaYlSIyD8jEutNuLW10qAkdYkIppYJcsFwaUkopVQdNBEopFeQ0ESilVJDTRKCUUkFOE4FSSgU5TQRKeYlIpYhk+bya7IlaEekpIhubantKNaWgeI5AKT/9YIwZ6nQQSjU3PSNQqgEisltEfi8iG0RklYj09s7vKSKLRWS9iHwpIsne+d1E5H0RWed9VQ1L4BaRv3nHt/9cRNp52z/srQ2xXkTmOPRnqiCmiUCp09rVuDQ0yWfZMWNMGvAS1milAC8CbxhjzgVmAS94578AfGWMGYI1Tk/VU+x9gBnGmEHAUeBG7/wngWHe7dxv1x+nVF30yWKlvETkhDEmqpb5u4FLjTE7vYP17TfGdBaRQqC7MabcO3+fMSZORAqARGNMqc82egL/Msb08X5+Agg1xjwnIp8BJ4APgA+MMSds/lOVqkbPCJTyj6ljOhClPtOVnO6juwargt5wYLW3CIpSzUYTgVL+meTzvtw7/R2nSxdOAb72Tn8J/BRO1UKOrWujIuICkowxS4AngFjgjLMSpeykRx5KndbOZzRWsOr2Vt1C2lFE1mMd1U/2zpuOVcnrcayqXlWjdD4CzBSRu7CO/H+KVeGqNm7gbW+yEOCFIC8NqRygfQRKNcDbR5BujCl0Ohal7KCXhpRSKsjpGYFSSgU5PSNQSqkgp4lAKaWCnCYCpZQKcpoIlFIqyGkiUEqpIPf/AbHqkHIqw0MZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_graphs(history, string):\n",
    "  plt.plot(history.history[string])\n",
    "  plt.plot(history.history['val_'+string])\n",
    "  plt.xlabel(\"Epochs\")\n",
    "  plt.ylabel(string)\n",
    "  plt.legend([string, 'val_'+string])\n",
    "  plt.show()\n",
    "  \n",
    "plot_graphs(history, \"accuracy\")\n",
    "plot_graphs(history, \"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.2909015e-05 9.8399794e-01 1.0172065e-03 5.7833669e-05 1.3471981e-02\n",
      "  1.4321870e-03]] bussiness\n"
     ]
    }
   ],
   "source": [
    "txt = [\"A WeWork shareholder has taken the company to court over the near-$1.7bn (£1.3bn) leaving package approved for ousted co-founder Adam Neumann.\"]\n",
    "seq = tokenizer.texts_to_sequences(txt)\n",
    "padded = pad_sequences(seq, maxlen=max_length)\n",
    "pred = model.predict(padded)\n",
    "labels = ['sport', 'bussiness', 'politics', 'tech', 'entertainment']\n",
    "print(pred, labels[np.argmax(pred)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
